{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy.sparse as ss\n",
    "import scipy.io as sio\n",
    "\n",
    "#保存数据\n",
    "import cPickle\n",
    "\n",
    "#event的特征需要编码\n",
    "from utils import FeatureEng\n",
    "from sklearn.preprocessing import normalize\n",
    "#距离\n",
    "import scipy.spatial.distance as ssd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of records :3137972\n"
     ]
    }
   ],
   "source": [
    "#读取数据，并统计有多少不同的events\n",
    "lines = 0\n",
    "fin = open(\"events.csv\", 'rb')\n",
    "#字段：event_id, user_id,start_time, city, state, zip, country, lat, and lng， 101 columns of words count\n",
    "fin.readline() # skip header，列名行\n",
    "for line in fin:\n",
    "    cols = line.strip().split(\",\")\n",
    "    lines += 1\n",
    "fin.close()\n",
    "\n",
    "print(\"number of records :%d\" % lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13418\n"
     ]
    }
   ],
   "source": [
    "#读取训练集和测试集中出现过的活动列表\n",
    "elist = cPickle.load(open(\"PE_eventIndex.pkl\", 'rb'))\n",
    "numberevents = len(elist)\n",
    "print(numberevents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = FeatureEng()\n",
    "\n",
    "fin = open(\"events.csv\", 'rb')\n",
    "#字段：event_id, user_id,start_time, city, state, zip, country, lat, and lng， 101 columns of words count\n",
    "fin.readline() # skip header\n",
    "#start_time, city, state, zip, country, lat, and lng\n",
    "eventPropMatrix = ss.dok_matrix((numberevents, 7))\n",
    "#词频特征\n",
    "eventContMatrix = ss.dok_matrix((numberevents, 101))\n",
    "for line in fin.readlines():\n",
    "    cols = line.strip().split(\",\")\n",
    "    eventId = str(cols[0])\n",
    "    \n",
    "    if elist.has_key(eventId):  #在训练集或测试集中出现\n",
    "        i = elist[eventId]\n",
    "        eventPropMatrix[i, 0] = f.getJoinedYearMonth(cols[2]) # start_time\n",
    "        eventPropMatrix[i, 1] = f.getFeatureHash(cols[3]) # city\n",
    "        eventPropMatrix[i, 2] = f.getFeatureHash(cols[4]) # state\n",
    "        eventPropMatrix[i, 3] = f.getFeatureHash(cols[5]) # zip\n",
    "        eventPropMatrix[i, 4] = f.getFeatureHash(cols[6]) # country\n",
    "        eventPropMatrix[i, 5] = f.getFloatValue(cols[7]) # lat\n",
    "        eventPropMatrix[i, 6] = f.getFloatValue(cols[8]) # lon\n",
    "        \n",
    "        #词频\n",
    "        for j in range(9, 109):\n",
    "            eventContMatrix[i, j-9] = cols[j]\n",
    "fin.close()\n",
    "#用L2模归一化\n",
    "eventPropMatrix = normalize(eventPropMatrix,\n",
    "    norm=\"l2\", copy=False)\n",
    "sio.mmwrite(\"EV_eventPropMatrix\", eventPropMatrix)\n",
    "#词频，可以考虑我们用这部分特征进行聚类，得到活动的genre\n",
    "eventContMatrix = normalize(eventContMatrix,\n",
    "    norm=\"l2\", copy=False)\n",
    "sio.mmwrite(\"EV_eventContMatrix\", eventContMatrix)\n",
    "# calculate 相似度 between event pairs based on the two matrices\n",
    "eventPropSim = ss.dok_matrix((numberevents, numberevents))\n",
    "eventContSim = ss.dok_matrix((numberevents, numberevents))   \n",
    "#读取在测试集和训练集中出现的活动对\n",
    "uniqueEventPairs = cPickle.load(open(\"PE_uniqueEventPairs.pkl\", 'rb'))\n",
    "\n",
    "for e1, e2 in uniqueEventPairs:\n",
    "    #i = eventIndex[e1]\n",
    "    #j = eventIndex[e2]\n",
    "    i = e1\n",
    "    j = e2\n",
    "    \n",
    "    #非词频特征，correlation相似度 （1 - correlation distance）\n",
    "    #注意：scipy.spatial.distance中correlation返回的是距离，即1-相关系数，取值范围[0,2]\n",
    "    #这里我们用的是相似度，将距离再还原成相似度\n",
    "    if not eventPropSim.has_key((i,j)):\n",
    "        epsim = 1 - ssd.correlation(eventPropMatrix.getrow(i).todense(),\n",
    "            eventPropMatrix.getrow(j).todense())\n",
    "        \n",
    "        eventPropSim[i, j] = epsim\n",
    "        eventPropSim[j, i] = epsim\n",
    "    \n",
    "    #对词频特征，Cosine 相似度（1 - cosnie distance）\n",
    "    if not eventContSim.has_key((i,j)):\n",
    "        ec_dist = 1 - ssd.cosine(eventContMatrix.getrow(i).todense(),\n",
    "            eventContMatrix.getrow(j).todense())\n",
    "    \n",
    "        eventContSim[i, j] = ec_dist\n",
    "        eventContSim[j, i] = ec_dist\n",
    "    \n",
    "sio.mmwrite(\"EV_eventPropSim\", eventPropSim)\n",
    "sio.mmwrite(\"EV_eventContSim\", eventContSim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eventPropSim.getrow(0).todense()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
